package com.atguigu.app;

import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

import java.util.HashSet;

/**
 * 读取端口数据,拆分成单个单词,对单词进行去重输出(即某个单词被写出过一次,以后就不再写出)。
 */
public class Flink07 {

    public static void main(String[] args) throws Exception {

        //1.获取执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(2);

        //2.读取端口数据
        DataStreamSource<String> socketTextStream = env.socketTextStream("hadoop102", 9999);

        //3.拆分单词
        SingleOutputStreamOperator<String> wordDS = socketTextStream.flatMap(new FlatMapFunction<String, String>() {
            @Override
            public void flatMap(String value, Collector<String> out) throws Exception {
                String[] words = value.split(" ");
                for (String word : words) {
                    out.collect(word);
                }
            }
        });

        //4.过滤
        SingleOutputStreamOperator<String> result = wordDS
                .keyBy(word -> word)
                .filter(new FilterFunction<String>() {

                    HashSet<String> hashSet = new HashSet<>();

                    @Override
                    public boolean filter(String value) throws Exception {

                        //数据已存在
                        if (hashSet.contains(value)) {
                            return false;
                        }

                        //数据不存在
                        hashSet.add(value);
                        return true;
                    }
                });

        //5.打印
        result.print();

        //6.启动
        env.execute();

    }

}
